/*
 * Copyright (c) 2018-2024 北京华恒盛世科技有限公司
 * QUICKPOOL is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *     http://license.coscl.org.cn/MulanPSL2
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
*/

#include "subcmd.h"

#define NAME_F 15
#define STAT_F 11
#define NUM_WIDE 7

static json *hostinfo_2json(struct QuickPoolHostInfo *, struct QuickPoolResourceReply *sharedres,
                            struct quickpoolInfo *);
static json *gpuinfo_2json(struct QuickPoolHostInfo *);
static json *hostnuma_2json(struct QuickPoolHostInfo *);

static void listnode_usage(const char *);
static void addnode_usage(const char *);
static void removenode_usage(const char *);
static int compare_node(const void *x, const void *y);

/**
 * @brief       qp node list handler, to query node information from js.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 *
 * @retval      0             #1: succeed
 * @retval      1             #2: input option is wrong or kind of input argument not specified
 * @retval      2             #3: argument has syntax error
 * @retval      3             #4: failed to query node information
 * @retval      4             #5: not enough memory to print json string
 */
int
cmd_list_nodes(int argc, char **argv)
{
    char *strResReq=NULL;
    bool_t gOpt = FALSE, tOpt = FALSE;
    bool_t fOpt = FALSE, wOpt = FALSE;
    while (TRUE) {
        int option_index = 0;
        static struct option long_options[] = {
            {"help", no_argument, 0, 'h'},
            {"requirement", required_argument, 0, 'R'},
            {"gpu", no_argument, 0, 'g'},
            {"topology", no_argument, 0, 't'},
            {"format", no_argument, 0, 'f'},
            {"wide", no_argument, 0, 'w'},
            {NULL, 0, 0, 0}
        };

        int opt;
        opt = getopt_long(argc, argv, "hR:fwgt", long_options, &option_index);
        if (-1 == opt) {
            break;
        }

        switch (opt) {
        case 'R':
            if (gOpt) {
                fprintf(stderr, "Cannot specify -g/--gpu option together with -R/--requirement.\n");
                listnode_usage(argv[0]);
                exit(1);
            }

            if (tOpt) {
                fprintf(stderr, "Cannot specify -t/--topology option together with -R/--requirement.\n");
                listnode_usage(argv[0]);
                exit(1);
            }

            strResReq = optarg;
            break;
        case 'g':
            if (NULL != strResReq) {
                fprintf(stderr, "Cannot specify -R/--requirement option together with -g/--gpu.\n");
                listnode_usage(argv[0]);
                exit(1);
            }

            if (tOpt) {
                fprintf(stderr, "Cannot specify -t/--topology option together with -g/--gpu.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            if (fOpt) {
                fprintf(stderr, "Cannot specify -f/--format option together with -g/--gpu.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            if (wOpt) {
                fprintf(stderr, "Cannot specify -w/--wide option together with -g/--gpu.\n");
                listnode_usage(argv[0]);
                exit(1);
            }

            gOpt = TRUE;
            break;
        case 't':
            if (NULL != strResReq) {
                fprintf(stderr, "Cannot specify -R/--requirement option together with -t/--topology.\n");
                listnode_usage(argv[0]);
                exit(1);
            }

            if (gOpt) {
                fprintf(stderr, "Cannot specify -g/--gpu option together with -t/--topology.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            if (fOpt) {
                fprintf(stderr, "Cannot specify -f/--foramt option together with -t/--topology.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            if (wOpt) {
                fprintf(stderr, "Cannot specify -w/--wide option together with -t/--topology.\n");
                listnode_usage(argv[0]);
                exit(1);
            }

            tOpt = TRUE;
            break;
        case 'f':
            if (gOpt) {
                fprintf(stderr, "Cannot specify -g/--gpu option together with -f/--format.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            if (tOpt) {
                fprintf(stderr, "Cannot specify -t/--topology option together with -f/--format.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            fOpt = TRUE;
            break;
        case 'w':
            if (gOpt) {
                fprintf(stderr, "Cannot specify -g/--gpu option together with -w/--wide.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            if (tOpt) {
                fprintf(stderr, "Cannot specify -t/--topology option together with -w/--wide.\n");
                listnode_usage(argv[0]);
                exit(1);
            }
            wOpt = TRUE;
            break;
        case 'h':
            listnode_usage(argv[0]);
            exit(0);
        case '?':
        default:
            listnode_usage(argv[0]);
            exit(1);
        }
    }

    // if not specify node name, set all to TRUE
    int all=FALSE;
    unsigned int numNode;
    char **nodes=NULL, **nodePoint;
    numNode = get_argument_names(argc, argv, optind, &nodes, &all, "host");
    if ((all && numNode == 1) || !numNode) {
        nodePoint = NULL;
    } else {
        nodePoint = nodes;
    }

    struct quickpoolInfo *qpInfo;
    if ((qpInfo = qp_info(NULL)) == NULL) {
	exit(3);
    }

    struct request nodeRequest;
    memset(&nodeRequest, 0, sizeof(struct request));

    int options=0, n=0;
    if (0 == numNode || NULL == nodePoint) {
        if (NULL != strResReq) {
            nodeRequest.number = 4;
        } else {
            nodeRequest.number = 3;
        }
        nodeRequest.keyvalues = (struct keyvalue *)calloc(nodeRequest.number, sizeof(struct keyvalue));
        if (NULL == nodeRequest.keyvalues) {
            fprintf(stderr, "calloc(%ld) failed due to %m.\n",
                    nodeRequest.number*sizeof(struct keyvalue));
            exit(4);
        }

        nodeRequest.keyvalues[n].key = REQUEST_KEY_OPTIONS;
        nodeRequest.keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == nodeRequest.keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            exit(4);
        }

        if (NULL != strResReq) {
            options |= CHECK_HOST;
            snprintf(nodeRequest.keyvalues[n].value, 12, "%d", options);
        } else {
            snprintf(nodeRequest.keyvalues[n].value, 12, "%d", 0);
        }
        n ++;

        nodeRequest.keyvalues[n].key = REQUEST_KEY_NUMBER;
        nodeRequest.keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == nodeRequest.keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            exit(4);
        }
        snprintf(nodeRequest.keyvalues[n].value, 12, "%d", 0);
        n ++;

        nodeRequest.keyvalues[n].key = REQUEST_KEY_NAMES;
        nodeRequest.keyvalues[n].value = copy_string(qp_getmyhostname());
        n ++;
    } else {

        // to query specific nodes.
        if (NULL != strResReq) {
            nodeRequest.number = 4;
        } else {
            nodeRequest.number = 3;
        }
        nodeRequest.keyvalues = (struct keyvalue *)calloc(nodeRequest.number,
                                                          sizeof(struct keyvalue));
        if (NULL == nodeRequest.keyvalues) {
            fprintf(stderr, "calloc(%ld) failed due to %m.\n",
                    nodeRequest.number*sizeof(struct keyvalue));
            exit(4);
        }

        nodeRequest.keyvalues[n].key = REQUEST_KEY_OPTIONS;
        nodeRequest.keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == nodeRequest.keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            exit(4);
        }
        snprintf(nodeRequest.keyvalues[n].value, 12, "%d", options);
        n ++;

        nodeRequest.keyvalues[n].key = REQUEST_KEY_NUMBER;
        nodeRequest.keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == nodeRequest.keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            exit(4);
        }
        snprintf(nodeRequest.keyvalues[n].value, 12, "%d", numNode);
        n ++;

        nodeRequest.keyvalues[n].key = REQUEST_KEY_NAMES;
        unsigned int i, size = 0;
        for (i = 0; i < numNode; i++) {
            size += strlen(nodePoint[i]) + 1;
        }
        nodeRequest.keyvalues[n].value = (char *)calloc(size, sizeof(char));
        if (NULL == nodeRequest.keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", size);
            exit(4);
        }

        for (i = 0; i < numNode; i++) {
            if (i == 0) {
                strcpy(nodeRequest.keyvalues[n].value, nodePoint[i]);
            } else {
                strcat(nodeRequest.keyvalues[n].value, " ");
                strcat(nodeRequest.keyvalues[n].value, nodePoint[i]);
            }
        }
        n ++;
    }

    if (NULL != strResReq) { // to select node satisfied with this requirement
        nodeRequest.keyvalues[n].key = REQUEST_KEY_RESREQ;
        nodeRequest.keyvalues[n].value = copy_string(strResReq);
        n ++;
    }

    FREEUP(nodes);

    quickpool_errno = 0;
    struct QuickPoolHostReply *hostReply;
    hostReply = client_get_hostinfo(&nodeRequest, NULL);
    if (quickpool_errno != 0 || NULL == hostReply) {

        if (NULL != hostReply) {
            // JS replied error message.
            fprintf(stderr, "%s\n", hostReply->error_message);
        } else {
            fprintf(stderr, "Fail to query node list due to above error.\n");
        }

        gf_xdr_free(gf_xdr_request, &nodeRequest);
        gf_xdr_free(libxdr_host_reply, hostReply);
        FREEUP(hostReply);

        exit(3);
    }

    gf_xdr_free(gf_xdr_request, &nodeRequest);

    qsort((struct QuickPoolHostInfo *)hostReply->hosts, hostReply->num_hosts, sizeof(struct QuickPoolHostInfo), compare_node);

    struct QuickPoolResourceReply *resourceReply=NULL;
    if (!gOpt && !tOpt) { // do not get resource info from js if --gpu or --topology is specified. 

        quickpool_errno = 0;

        struct request resourceRequest;
        memset(&resourceRequest, 0, sizeof(struct request));

        resourceRequest.number = 1;
        resourceRequest.keyvalues = (struct keyvalue *)calloc(1, sizeof(struct keyvalue));
        if (NULL == resourceRequest.keyvalues) {
            fprintf(stderr, "calloc(%ld) failed due to %m.\n", sizeof(struct keyvalue));
            exit(4);
        }

        resourceRequest.keyvalues[0].key = REQUEST_KEY_OPTIONS;
        resourceRequest.keyvalues[0].value = (char *)calloc(12, sizeof(char));
        if (NULL == resourceRequest.keyvalues[0].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            exit(4);
        }
        snprintf(resourceRequest.keyvalues[0].value, 12, "%d", 0);

        resourceReply = client_get_resourceinfo(&resourceRequest, NULL);
        if (NULL == resourceReply || quickpool_errno != 0) {

            if (NULL == resourceReply) {
                fprintf(stderr, "Fail to query resource info due to above error.\n");
                gf_xdr_free(gf_xdr_request, &resourceRequest);
                exit(3);
            } else if (quickpool_errno != ERROR_NO_RESOURCE) {
                // JS replied error message.
                fprintf(stderr, "%s\n", resourceReply->error_message);

                gf_xdr_free(gf_xdr_request, (void *)&resourceRequest);
                gf_xdr_free(libxdr_resource_reply, resourceReply);
                FREEUP(resourceReply);

                exit(3);
            }
        }

        gf_xdr_free(gf_xdr_request, (void *)&resourceRequest);
    }

    unsigned int k;
    if (gOpt) { // --gpu option, only display node gpu information.

        json *jsonHost = json_CreateObject();
        for (k = 0; k < hostReply->num_hosts; k++) {

            json *jsonHostGPU = gpuinfo_2json(&(hostReply->hosts[k]));
            if (NULL == jsonHostGPU) {
                continue;
            }

            json_AddStringToObject(jsonHost, "HOST_NAME", hostReply->hosts[k].host);
            json_AddItemToObject(jsonHost, "GPU", jsonHostGPU);
        }

        if (0 < json_GetArraySize(jsonHost)) {
            char *strJson;
            strJson = json_Print(jsonHost);
            if (NULL != strJson) {
                fprintf(stdout, "%s\n", strJson);
                FREEUP(strJson);
            } else {
                perror("malloc");
                gf_xdr_free(libxdr_host_reply, hostReply);
                FREEUP(hostReply);
                json_Delete(jsonHost);
                exit(4);
            }
        } else {
            fprintf(stderr, "Cluster has no node with GPU.\n");
        }

        gf_xdr_free(libxdr_host_reply, hostReply);
        FREEUP(hostReply);
        json_Delete(jsonHost);

        exit(0);
    } else if (tOpt) {

        json *jsonHost = json_CreateObject();
        for (k = 0; k < hostReply->num_hosts; k++) {

            json *jsonHostNuma = hostnuma_2json(&(hostReply->hosts[k]));
            if (NULL == jsonHostNuma) {
                continue;
            }

            json_AddStringToObject(jsonHost, "HOST_NAME", hostReply->hosts[k].host);

            double memory;
            char *strUnit;
            if (hostReply->hosts[k].numa->memory > 1024*1024) {
                memory = (double)(hostReply->hosts[k].numa->memory)/(double)(1024*1024);
                strUnit = (char *)"T";
            } else if (hostReply->hosts[k].numa->memory > 1024) {
                memory = (double)(hostReply->hosts[k].numa->memory)/(double)(1024);
                strUnit = (char *)"G";
            } else {
                strUnit = (char *)"M";
                memory = hostReply->hosts[k].numa->memory;
            }

            char strMem[56+1];
            snprintf(strMem, 56, "%.1f%s", memory, strUnit);
            json_AddStringToObject(jsonHost, "TOTAL_MEM", strMem);

            json_AddItemToObject(jsonHost, "NUMA_INFO", jsonHostNuma);
        }

        if (0 < json_GetArraySize(jsonHost)) {
            char *strJson;
            strJson = json_Print(jsonHost);
            if (NULL != strJson) {
                fprintf(stdout, "%s\n", strJson);
                FREEUP(strJson);
            } else {
                perror("malloc");
                gf_xdr_free(libxdr_host_reply, hostReply);
                FREEUP(hostReply);
                json_Delete(jsonHost);
                exit(4);
            }
        } else {
            fprintf(stdout, "Cluster has no node with numa.\n");
        }

        gf_xdr_free(libxdr_host_reply, hostReply);
        FREEUP(hostReply);
        json_Delete(jsonHost);

        exit(0);
    }
    
    if ((fOpt | wOpt) && 0 < hostReply->num_hosts) {
        char stat[128];
        char buf[4096+1];
        char header[4096+1];
        snprintf(buf, 4096, "%%-%ds  %%-%ds  %%%dd  %%%dd  %%%dd  %%%dd  %%%dd  %%%dd\n",
                 NAME_F, STAT_F, NUM_WIDE, NUM_WIDE, NUM_WIDE, NUM_WIDE, NUM_WIDE, NUM_WIDE);
        snprintf(header, 4096, "%%-%ds  %%-%ds  %%%ds  %%%ds  %%%ds  %%%ds  %%%ds  %%%ds\n",
                 NAME_F, STAT_F, NUM_WIDE, NUM_WIDE, NUM_WIDE, NUM_WIDE, NUM_WIDE, NUM_WIDE);
        fprintf(stdout,header, "NAME", "STAT", "MAX", "TOTAL", "RUN", "USUSP", "SSUSP", "RSV");
        for (k = 0; k < hostReply->num_hosts; k++) {
            get_hoststatus(&hostReply->hosts[k],stat);
            if (strlen(hostReply->hosts[k].host) > NAME_F && wOpt == FALSE) {
                hostReply->hosts[k].host[NAME_F-1] = '*';
                hostReply->hosts[k].host[NAME_F] = '\0';
            } else {
                fprintf(stdout,buf,
                hostReply->hosts[k].host,stat,hostReply->hosts[k].max_jobs,hostReply->hosts[k].num_job_slot,
                hostReply->hosts[k].num_running_slot,hostReply->hosts[k].num_USUSP_slot,hostReply->hosts[k].num_SSUSP_slot,
                hostReply->hosts[k].num_reserving_slot);
                continue;
            }
            fprintf(stdout,buf,
            hostReply->hosts[k].host,stat,hostReply->hosts[k].max_jobs,hostReply->hosts[k].num_job_slot,
            hostReply->hosts[k].num_running_slot,hostReply->hosts[k].num_USUSP_slot,hostReply->hosts[k].num_SSUSP_slot,
            hostReply->hosts[k].num_reserving_slot);
        }
        
        FREEUP(hostReply);
        exit(0);
    }

    json *jsonNodeList = json_CreateObject();
    for (k = 0; k < hostReply->num_hosts; k++) {

        json *jsonHost = hostinfo_2json(&(hostReply->hosts[k]), resourceReply, qpInfo);
        if (NULL == jsonHost) {
            continue;
        }

        json_AddItemToObject(jsonNodeList, hostReply->hosts[k].host, jsonHost);
    }

    if (0 < json_GetArraySize(jsonNodeList)) {
        char *strJson;
        strJson = json_Print(jsonNodeList);
        if (NULL != strJson) {
            fprintf(stdout, "%s\n", strJson);
            FREEUP(strJson);
        } else {

            perror("malloc");
            json_Delete(jsonNodeList);
            gf_xdr_free(libxdr_resource_reply, resourceReply);
            FREEUP(resourceReply);
            gf_xdr_free(libxdr_host_reply, hostReply);
            FREEUP(hostReply);

            exit(4);
        }
    } else {
        // nothing in json means there's no node in the cluster, should not go here.
        fprintf(stderr, "No node information in the cluster.\n");
    }
    json_Delete(jsonNodeList);

    gf_xdr_free(libxdr_resource_reply, resourceReply);
    FREEUP(resourceReply);
    gf_xdr_free(libxdr_host_reply, hostReply);
    FREEUP(hostReply);

    exit(0);
} // end function cmd_list_nodes

/**
 * @brief       qp node add handler, to add node to cluster.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 *
 * @retval      0             #1: succeed
 * @retval      1             #2: input option is wrong or kind of input argument not specified
 * @retval      2             #3: argument has syntax error
 * @retval      3             #4: failed to add node
 * @retval      4             #5: not enough memory
 */
int
cmd_add_node(int argc, char **argv)
{
    struct request req;
    memset(&req, 0, sizeof(struct request));

    req.number = 7;
    req.keyvalues = (struct keyvalue *)calloc(req.number, sizeof(struct keyvalue));
    if (NULL == req.keyvalues) {
        perror("calloc");
        exit(4);
    }

    char *strModel=NULL, *strType=NULL, *strRunWindow=NULL, *strDispWindow=NULL;
    char *strResource=NULL;
    int maxJobs = -1;
    while (TRUE) {
        int option_index = 0;
        static struct option long_options[] = {
            {"help", no_argument, 0, 'h'},
            {"model", required_argument, 0, 'm'},
            {"type", required_argument, 0, 't'},
            {"run_window", required_argument, 0, 'w'},
            {"dispatch_window", required_argument, 0, 'D'},
            {"resource", required_argument, 0, 'R'},
            {"maxjob", required_argument, 0, 'M'},
            {NULL, 0, 0, 0}
        };

        int opt;
        opt = getopt_long(argc, argv, "hm:t:w:D:R:M:", long_options, &option_index);
        if (-1 == opt) {
            break;
        }

        switch (opt) {
        case 'm':
            if (NULL != strModel) {
                fprintf(stderr, "Do not specify -m/--model option for multiple times.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Specify a model name with -m/--model option.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            strModel = copy_string(optarg);
            break;
        case 't':
            if (NULL != strType) {
                fprintf(stderr, "Do not specify -t/--type option for multiple times.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Specify a node type name with -t/--type option.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            strType = copy_string(optarg);
            break;
        case 'w':
            if (NULL != strRunWindow) {
                fprintf(stderr, "Do not specify -w/--run_window option for multiple times.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Specify time window with -w/--run_window option.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            strRunWindow = copy_string(optarg);
            break;
        case 'D':
            if (NULL != strDispWindow) {
                fprintf(stderr, "Do not specify -D/--dispatch_window option for multiple times.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Specify time window with -D/--dispatch_window option.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            strDispWindow = copy_string(optarg);
            break;
        case 'R':
            if (NULL != strResource) {
                fprintf(stderr, "Do not specify -R/--resource option for multiple times.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Specify resource list with -R/--resource option.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            strResource = copy_string(optarg);
            break;
        case 'M':
            if (0 <= maxJobs) {
                fprintf(stderr, "Do not specify -M/--maxjob option for multiple times.\n");
                addnode_usage(argv[0]);
                exit(1);
            }

            maxJobs = gf_atoi(optarg, INFINIT_INT, -1);
            if (INFINIT_INT == maxJobs) {
                fprintf(stderr, "%s: Must specify a non-negative integer for -M/--maxjob option.",
                        optarg);
                exit(2);
            }

            break;
        case 'h':
            addnode_usage(argv[0]);
            exit(0);
        case '?':
        default:
            addnode_usage(argv[0]);
            exit(1);
        }
    }

    if (NULL == argv[optind]) {
        fprintf(stderr, "Must specify a node name to be added.\n");
        addnode_usage(argv[0]);
        exit(1);
    }

    char *strNode = copy_string(argv[optind]);

    unsigned int num=0;
    req.keyvalues[num].key = REQUEST_KEY_HOSTNAME;
    req.keyvalues[num].value = strNode;
    num ++;

    if (NULL != strModel) {
        req.keyvalues[num].key = REQUEST_KEY_HOST_MODEL;
        req.keyvalues[num].value = strModel;
        num ++;
    }

    if (NULL != strType) {
        req.keyvalues[num].key = REQUEST_KEY_HOST_TYPE;
        req.keyvalues[num].value = strType;
        num ++;
    }

    if (NULL != strRunWindow) {
        req.keyvalues[num].key = REQUEST_KEY_RUN_WINDOW;
        req.keyvalues[num].value = strRunWindow;
        num ++;
    }

    if (NULL != strDispWindow) {
        req.keyvalues[num].key = REQUEST_KEY_DISPATCH_WINDOW;
        req.keyvalues[num].value = strDispWindow;
        num ++;
    }

    if (NULL != strResource) {
        req.keyvalues[num].key = REQUEST_KEY_RESOURCE;
        req.keyvalues[num].value = strResource;
        num ++;
    }

    if (0 <= maxJobs) {
        req.keyvalues[num].key = REQUEST_KEY_MAXJOB;
        req.keyvalues[num].value = (char *)calloc(12, sizeof(char));
        if (NULL == req.keyvalues[num].value) {
            gf_xdr_free(gf_xdr_request, &req);
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            exit(4);
        }
        snprintf(req.keyvalues[num].value, 12, "%d", maxJobs);
        num ++;
    }

    req.number = num;

    int ret;
    ret = client_addhost(&req, NULL);
    if (0 > ret) {

        gf_xdr_free(gf_xdr_request, &req);
        exit(3);
    }

    printf("Node <%s> is added to the cluster.\n", strNode);
    gf_xdr_free(gf_xdr_request, &req);

    exit(0);
} // end function cmd_add_node

/**
 * @brief       qp node remove handler, to remove node from cluster.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 *
 * @retval      0             #1: succeed
 * @retval      1             #2: input option is wrong or kind of input argument not specified
 * @retval      3             #3: failed to remove node
 */
int
cmd_remove_node(int argc, char **argv)
{
    while (TRUE) {
        int option_index = 0;
        static struct option long_options[] = {
            {"help", no_argument, 0, 'h'},
            {NULL, 0, 0, 0}
        };

        int opt;
        opt = getopt_long(argc, argv, "h", long_options, &option_index);
        if (-1 == opt) {
            break;
        }

        switch (opt) {
        case 'h':
            removenode_usage(argv[0]);
            exit(0);
        case '?':
        default:
            removenode_usage(argv[0]);
            exit(1);
        }
    }

    if (NULL == argv[optind]) {
        fprintf(stderr, "Must specify a node name to be removeed.\n");
        removenode_usage(argv[0]);
        exit(1);
    }

    int ret;
    ret = client_removehost(argv[optind], NULL);
    if (0 > ret) {
        exit(3);
    }

    printf("Node <%s> was removed from the cluster.\n", argv[optind]);

    exit(0);
} // end function cmd_remove_node

/* ------------------------------------------------
 *
 *  below are static functions used in this file
 *
 * ------------------------------------------------ */

static void
get_closed_status(int instatus, char *outstatus)
{
    if (instatus & NODE_STATUS_DISABLED) {
        strcpy(outstatus,"closed_Adm");
    } else if (instatus & NODE_STATUS_EXCLUSIVE) {
        strcpy(outstatus,"closed_Excl");
    } else if (instatus & NODE_STATUS_WIND) {
        strcpy(outstatus,"closed_Wind");
    } else if (instatus & NODE_STATUS_FULL) {
        strcpy(outstatus,"closed_Full");
    } else if (instatus & NODE_STATUS_BUSY) {
        strcpy(outstatus,"closed_Busy");
    } else {
        strcpy(outstatus,"unknown");
    }

    return;
} // end function get_closed_status

/**
 * @brief       transform host structure to json.
 *
 * @param[in]   host          #1: host to be tranformed
 * @param[in]   sharedres     #2: if host has shared resources, transfer them
 * @param[in]   info          #3: load index and resource table
 *
 * @retval      not NULL      #1: transformed json object
 *
 * @note        if host is unavail, do not transfer load
 */
static json *
hostinfo_2json(struct QuickPoolHostInfo *host, struct QuickPoolResourceReply *sharedres,
               struct quickpoolInfo *info)
{
    json *jsonHost = json_CreateObject();

    s2j_json_set_basic_element(jsonHost, host, string, "HOST_NAME", host);
    
    char strStatus[16];
    get_hoststatus(host,strStatus);
    
    json_AddStringToObject(jsonHost, "STATUS", strStatus);

    if (host->max_jobs < INFINIT_INT) {
        s2j_json_set_basic_element(jsonHost, host, int, "MAX", max_jobs);
    }

    if (host->user_slot_limit < INFINIT_INT) {
        s2j_json_set_basic_element(jsonHost, host, int, "UJOB_LIMIT", user_slot_limit);
    }

    s2j_json_set_basic_element(jsonHost, host, int, "NSLOTS", num_job_slot);
    s2j_json_set_basic_element(jsonHost, host, int, "RUN", num_running_slot);
    s2j_json_set_basic_element(jsonHost, host, int, "SSUSP", num_SSUSP_slot);
    s2j_json_set_basic_element(jsonHost, host, int, "USUSP", num_USUSP_slot);
    s2j_json_set_basic_element(jsonHost, host, int, "RSV", num_reserving_slot);

    if (NULL != host->dispatch_window && strlen(host->dispatch_window) > 0) {

        s2j_json_set_basic_element(jsonHost, host, string, "DISPATCH_WINDOW", dispatch_window);
    }

    if (host->mig < INFINIT_INT && host->mig > 0) {

        s2j_json_set_basic_element(jsonHost, host, int, "MIG", mig);
    }

    json *jsonLoadSchedule;
    jsonLoadSchedule = json_CreateObject();

    json *jsonLoadStop;
    jsonLoadStop = json_CreateObject();

    unsigned int i;
    for (i=0; i<info->num_index; i++) {
        char strLoadSchedule[64+1];
        char strLoadStop[64+1];
        strLoadSchedule[0] = '\0';
        strLoadStop[0] = '\0';

        if (host->load_schedule[i] < INFINIT_LOAD && host->load_schedule[i] > -INFINIT_LOAD) {
            snprintf(strLoadSchedule, 64, "%.2f", host->load_schedule[i]);
        }

        if (host->load_stop[i] < INFINIT_LOAD && host->load_stop[i] > -INFINIT_LOAD) {
            snprintf(strLoadStop, 64, "%.2f", host->load_stop[i]);
        }

        if (0 < strlen(strLoadSchedule)) {
            json_AddStringToObject(jsonLoadSchedule, info->resource_table[i].name, strLoadSchedule);
        }

        if (0 < strlen(strLoadStop)) {
            json_AddStringToObject(jsonLoadStop, info->resource_table[i].name, strLoadStop);
        }
    }

    if (0 < json_GetArraySize(jsonLoadSchedule)) {
        json_AddItemToObject(jsonHost, "LOAD_SCHED", jsonLoadSchedule);
    } else {
        json_Delete(jsonLoadSchedule);
    }

    if (0 < json_GetArraySize(jsonLoadStop)) {
        json_AddItemToObject(jsonHost, "LOAD_STOP", jsonLoadStop);
    } else {
        json_Delete(jsonLoadStop);
    }

    if (host->status & NODE_STATUS_UNAVAIL) { // host is unavail, stop at here.
        return jsonHost;
    }

    json *jsonLoad;
    jsonLoad = json_CreateObject();

    json *jsonRsv;
    jsonRsv = json_CreateObject();

    for (i=0; i<info->num_index; i++) { // load indicies
        char *strUnit;
        double loadVal;
        double rsvVal;
        char strLoad[56+1];

        char *busy;
        if ((host->status & NODE_STATUS_BUSY)
            && (NODE_ISBUSYON(host->busy_schedule, i) || NODE_ISBUSYON(host->busy_stop, i))) {
            busy = (char *)"*";
        } else {
            busy = EMPTY_STRING;
        }

        switch (i) {
        case TMP:
        case SWP:
        case MEM:
            loadVal = host->real_load[i];
            rsvVal = (host->load[i] >= loadVal)? (host->load[i] - loadVal):(loadVal - host->load[i]);

            if (INFINIT_LOAD > loadVal && 0 <= loadVal) {

                if (loadVal > 1024*1024) {
                    strUnit = (char *)"T";
                    loadVal = (loadVal)/(double)(1024*1024);
                } else if (loadVal > 1024) {
                    strUnit = (char *)"G";
                    loadVal = (loadVal)/(double)(1024);
                } else {
                    strUnit = (char *)"M";
                }

                snprintf(strLoad, 56, "%s%.1f%s", busy, loadVal, strUnit);
                json_AddStringToObject(jsonLoad, info->resource_table[i].name, strLoad);
            }

            if (INFINIT_LOAD > rsvVal && !FLOAT_EQUAL_ZERO(rsvVal)) {

                if (rsvVal > 1024*1024) {
                    strUnit = (char *)"T";
                    rsvVal = (rsvVal)/(double)(1024*1024);
                } else if (rsvVal > 1024) {
                    strUnit = (char *)"G";
                    rsvVal = (rsvVal)/(double)(1024);
                } else {
                    strUnit = (char *)"M";
                }

                snprintf(strLoad, 56, "%.1f%s", rsvVal, strUnit);
                json_AddStringToObject(jsonRsv, info->resource_table[i].name, strLoad);
            }

            break;
        default:
            loadVal = host->real_load[i];
            rsvVal = (host->load[i] >= loadVal)? (host->load[i] - loadVal):(loadVal - host->load[i]);

            if (INFINIT_LOAD > loadVal && 0 <= loadVal) {
                snprintf(strLoad, 56, "%s%.2f", busy, loadVal);
                json_AddStringToObject(jsonLoad, info->resource_table[i].name, strLoad);
            }

            if (INFINIT_LOAD > rsvVal && !FLOAT_EQUAL_ZERO(rsvVal)) {
                snprintf(strLoad, 56, "%.2f", rsvVal);
                json_AddStringToObject(jsonRsv, info->resource_table[i].name, strLoad);
            }

            break;
        }
    }

    unsigned int j, k;
    for (i=0; i<sharedres->num_resources; i++) { // shared resources

        bool_t found = FALSE;
        for (j=0; j<info->num_resources; j++) {
            if (strcmp(info->resource_table[j].name, sharedres->resources[i].resource_name) != 0) {
                continue;
            }

            if (info->resource_table[j].flags & RESOURCE_FLAG_DYNAMIC) {
                found = TRUE;
            }

            break;
        }

        if (!found) {
            continue;
        }

        for (j=0; j<sharedres->resources[i].num_instances; j++) {

            struct QuickPoolResourceMap *resMap = &(sharedres->resources[i].instances[j]);
            strip_spaces(resMap->total_value, FALSE);
            strip_spaces(resMap->reserve_value, FALSE);

            float rsvVal = atof(resMap->reserve_value);

            for (k=0; k<resMap->num_hosts; k++) {

                if (strcmp(resMap->hosts[k], ALL_STRING) == 0) {

                    s2j_json_set_basic_element(jsonLoad, resMap, string,
                                               sharedres->resources[i].resource_name, total_value);

                    if (INFINIT_LOAD > rsvVal && !FLOAT_EQUAL_ZERO(rsvVal)) {
                        s2j_json_set_basic_element(jsonRsv, resMap, string,
                                                   sharedres->resources[i].resource_name,
                                                   reserve_value);
                    }

                    break;
                }

                if (strcmp(resMap->hosts[k], host->host) == 0) {

                    s2j_json_set_basic_element(jsonLoad, resMap, string,
                                               sharedres->resources[i].resource_name, total_value);

                    if (INFINIT_LOAD > rsvVal && !FLOAT_EQUAL_ZERO(rsvVal)) {
                        s2j_json_set_basic_element(jsonRsv, resMap, string,
                                                   sharedres->resources[i].resource_name,
                                                   reserve_value);
                    }

                    break;
                }
            }
        }
    }

    if (0 < json_GetArraySize(jsonLoad)) {
        json_AddItemToObject(jsonHost, "LOAD_TOTAL", jsonLoad);
    } else {
        json_Delete(jsonLoad);
    }

    if (0 < json_GetArraySize(jsonRsv)) {
        json_AddItemToObject(jsonHost, "LOAD_RESERVED", jsonRsv);
    } else {
        json_Delete(jsonRsv);
    }

    json *jsonGPU = gpuinfo_2json(host);
    if (NULL != jsonGPU) {
        json_AddItemToObject(jsonHost, "GPU_INFO", jsonGPU);
    }

    return jsonHost;
} // end function hostinfo_2json

/**
 * @brief       transform host GPU information to json.
 *
 * @param[in]   host          #1: host whose GPU is being tranformed
 *
 * @retval      not NULL      #1: transformed json object
 * @retval      NULL          #2: host has no GPU
 */
static json *
gpuinfo_2json(struct QuickPoolHostInfo *host)
{
    if (0 == host->num_GPU) {
        return NULL;
    }

    json *jsonGPU = NULL;

#if defined(QUICKPOOL_WITH_CUDA)
    jsonGPU = json_CreateObject();

    s2j_json_set_basic_element(jsonGPU, host, int, "NGPUS", num_GPU);

    unsigned int j;
    for(j = 0; j < host->num_GPU; j++) {

        json *jsonStatic = json_CreateObject();
        struct hostGPUList *gpuStatic = &(host->gpuinfo_static[j]);
        s2j_json_set_basic_element(jsonStatic, gpuStatic, int, "ID", gpu_id);
        s2j_json_set_basic_element(jsonStatic, gpuStatic, string, "MODEL", gpu_model);
        s2j_json_set_basic_element(jsonStatic, gpuStatic, double, "FACTOR", gpu_factor);

        double memory;
        char *strUnit;
        if (1024*1024 < gpuStatic->gpu_maxmem) {
            memory = (double)(gpuStatic->gpu_maxmem)/(double)(1024*1024);
            strUnit = (char *)"T";
        } else if (1024 < gpuStatic->gpu_maxmem) {
            memory = (double)(gpuStatic->gpu_maxmem)/(double)(1024);
            strUnit = (char *)"G";
        } else {
            strUnit = (char *)"M";
            memory = gpuStatic->gpu_maxmem;
        }

        char strMem[56+1];
        snprintf(strMem, 56, "%.1f%s", memory, strUnit);
        json_AddStringToObject(jsonStatic, "MEM", strMem);

        json_AddItemToObject(jsonGPU, "STATIC_ATTRIBUTE", jsonStatic);

        json *jsonDynamic = json_CreateObject();
        struct hostGPULoad *gpuLoad = &(host->gpuinfo_dynamic[j]);
        s2j_json_set_basic_element(jsonDynamic, gpuLoad, int, "ID", gpu_id);

        char *strMode;
        if (gpuLoad->gpu_mode == GPU_COMPUTEMODE_DEFAULT) {
            strMode = DEFAULT_STRING;
        } else if (gpuLoad->gpu_mode == GPU_COMPUTEMODE_EXCLUSIVE) {
            strMode = (char *)"exclusive";
        } else if (gpuLoad->gpu_mode == GPU_COMPUTEMODE_PROHIBITED) {
            strMode = (char *)"prohibited";
        } else {
            strMode = (char *)"unknown";
        }
        json_AddStringToObject(jsonDynamic, "MODE", strMode);

        s2j_json_set_basic_element(jsonDynamic, gpuLoad, string, "STATUS", status);
        s2j_json_set_basic_element(jsonDynamic, gpuLoad, int, "NUM_JOB", num_job);
        s2j_json_set_basic_element(jsonDynamic, gpuLoad, int, "TEMPERATURE", gpu_temperature);
        s2j_json_set_basic_element(jsonDynamic, gpuLoad, double, "UT", gpu_ut);

        if (NULL != gpuLoad->error && 0 < strlen(gpuLoad->error)) {
            s2j_json_set_basic_element(jsonDynamic, gpuLoad, string, "ERROR", error);
        }

        if (1024*1024 < gpuLoad->avail_gpu_mem) {
            memory = (double)(gpuLoad->avail_gpu_mem)/(double)(1024*1024);
            strUnit = (char *)"T";
        } else if (1024 < gpuLoad->avail_gpu_mem) {
            memory = (double)(gpuLoad->avail_gpu_mem)/(double)(1024);
            strUnit = (char *)"G";
        } else {
            strUnit = (char *)"M";
            memory = gpuLoad->avail_gpu_mem;
        }

        snprintf(strMem, 56, "%.1f%s", memory, strUnit);
        json_AddStringToObject(jsonDynamic, "AVAIL_MEM", strMem);

        json_AddItemToObject(jsonGPU, "DYNAMIC_ATTRIBUTE", jsonDynamic);
    }

#endif    
    return jsonGPU;
} // end function gpuinfo_2json

static void
numaobj_2json(struct json *root, struct json *numa, struct json *socket, struct json *core,
              struct numa_obj *numaobj)
{
    for (; numaobj != NULL; numaobj = numaobj->back) {

        if (numaobj->type == NUMA_NODE) {

            numa = json_CreateObject();
            s2j_json_set_basic_element(numa, numaobj, int, "INDEX", index_sibling);

            double memory;
            char *strUnit;
            if (numaobj->memory > 1024*1024) {
                memory = (double)(numaobj->memory)/(double)(1024*1024);
                strUnit = (char *)"T";
            } else if (numaobj->memory > 1024) {
                memory = (double)(numaobj->memory)/(double)(1024);
                strUnit = (char *)"G";
            } else {
                memory = numaobj->memory;
                strUnit = (char *)"M";
            }

            char strMem[56+1];
            snprintf(strMem, 56, "%.1f%s", memory, strUnit);
            json_AddStringToObject(numa, "MEM", strMem);

            json_AddItemToObject(root, "NUMA", numa);
        }

        if (numaobj->type == NUMA_SOCKET) {

            socket = json_CreateObject();
            s2j_json_set_basic_element(socket, numaobj, int, "INDEX", index_sibling);
            json_AddItemToObject(numa, "SOCKET", socket);
        }

        if (numaobj->type == NUMA_CORE) {

            core = json_CreateObject();
            s2j_json_set_basic_element(core, numaobj, int, "INDEX", index);
            json_AddItemToObject(socket, "CORE", core);
        }

        numaobj_2json(root, numa, socket, core, numaobj->child);
    }
} // end function numaobj_2json

/**
 * @brief       transform host NUMA information to json.
 *
 * @param[in]   host          #1: host whose NUMA is being tranformed
 *
 * @retval      not NULL      #1: transformed json object
 * @retval      NULL          #2: host has no NUMA
 */
static json *
hostnuma_2json(struct QuickPoolHostInfo *host)
{
    if (NULL == host->numa) {
        return NULL;
    }

    json *jsonNuma = json_CreateObject();
    numaobj_2json(jsonNuma, NULL, NULL, NULL, host->numa);

    return jsonNuma;
} // end function hostnuma_2json

static void
listnode_usage(const char *cmd)
{
    fprintf(stdout, "qp node %s usage: ", cmd);
    fprintf(stdout, "[-h, --help] [-f,--format] [-w,--wide][-R, --requirement res_req] [host_name ...]\n");
    fprintf(stdout, "or\n");
    fprintf(stdout, "\t\t    [-g, --gpu] [host_name ...]\n");
    fprintf(stdout, "or\n");
    fprintf(stdout, "\t\t    [-t, --topology] [host_name ...]\n");

    return;
}

static void
addnode_usage(const char *cmd)
{
    fprintf(stdout, "qp node %s usage: [-h, --help]\n", cmd);
    fprintf(stdout, "\t\t   [-m, --model hostModel] [-t, --type hostType]\n");
    fprintf(stdout, "\t\t   [-w, --running window] [-D, --dispatch window]\n");
    fprintf(stdout, "\t\t   [-R, --resource static_resources] [-M, --maxjob MXJ] hostname\n");

    return;
}

static void
removenode_usage(const char *cmd)
{
    fprintf(stdout, "qp node %s usage: [-h, --help] hostname\n", cmd);

    return;
}

char *
get_hoststatus(struct QuickPoolHostInfo *host, char strStatus[])
{
    strcpy(strStatus,"ok");
    if (host->status & NODE_STATUS_UNAVAIL) {
        strcpy(strStatus,"unavail");
    } else if (host->status & (NODE_STATUS_BUSY | NODE_STATUS_WIND | NODE_STATUS_DISABLED
                               | NODE_STATUS_EXCLUSIVE | NODE_STATUS_FULL)) {
        get_closed_status(host->status, strStatus);
    }
    return strStatus;
}

static int
compare_node(const void *x, const void *y)
{
    struct QuickPoolHostInfo *node1;
    struct QuickPoolHostInfo *node2;
    node1 = (struct QuickPoolHostInfo *)x;
    node2 = (struct QuickPoolHostInfo *)y;

    return strcmp(node1->host,node2->host);
}
